import matplotlib.colors as colors
import matplotlib.pyplot as plt
import numpy as np
import plotly.graph_objects as go
from pathlib import Path
from PIL import Image
from scipy.spatial.transform import Rotation as R
import os

DATASET_ROOT = os.path.expanduser("~/dev/projectaria_tools/projectaria_tools_ase_data")  # Specify your own dataset path
SCENE_ID = 0  # Select a scene id

dataset_path = Path(DATASET_ROOT)
print("Chosen ASE data path: ", dataset_path)
print(f"Using Scene {SCENE_ID} for these examples")

scene_path = dataset_path / str(SCENE_ID)

Chosen ASE data path:  /Users/florian/dev/projectaria_tools/projectaria_tools_ase_data
Using Scene 0 for these examples

from code_snippets.readers import read_points_file, read_trajectory_file, read_language_file

# Load scene point cloud using read_points_file()
points_path = scene_path / "semidense_points.csv.gz"
points = read_points_file(points_path)

# Load a trajectory using read_trajectory_file() 
trajectory_path = scene_path / "trajectory.csv"
trajectory = read_trajectory_file(trajectory_path)

# Load a scene command language using read_language_file()
language_path = scene_path / "ase_scene_language.txt"
entities = read_language_file(language_path)

Loaded point cloud with 433426 points.
Loaded trajectory with 350 device poses.
Loaded scene commands with a total of 15 entities.

from code_snippets.interpreter import language_to_bboxes

# Interpret scene commands into 3D Boxes
entity_boxes = language_to_bboxes(entities)

Scene contains:
  wall: 8
  door: 3
  window: 4

def random_bright_colormap(num_colors=1024):
    bright_colors = np.random.rand(num_colors, 3)
    bright_colors = (bright_colors + 1) / 2
    return colors.ListedColormap([c for c in bright_colors])

scene_path = dataset_path / str(SCENE_ID)

rgb_dir = scene_path / "rgb"
depth_dir = scene_path / "depth"
instance_dir = scene_path / "instances"

# Load images
frame_idx = 209
frame_id = str(frame_idx).zfill(7)
rgb_path = rgb_dir / f"vignette{frame_id}.jpg"
depth_path = depth_dir / f"depth{frame_id}.png"
instance_path = instance_dir / f"instance{frame_id}.png"

rgb = Image.open(rgb_path)
depth = Image.open(depth_path)
instances = Image.open(instance_path)

# Note: Images are rotated to upright for visualization.
# However, the camera calibration is for the original orientation.
rgb_to_plot = rgb.rotate(-90, expand=True)
depth_to_plot = depth.rotate(-90, expand=True)
instances_to_plot = instances.rotate(-90, expand=True)

from projectaria_tools.projects import ase

def transform_3d_points(transform, points):
    N = len(points)
    points_h = np.concatenate([points, np.ones((N, 1))], axis=1)
    transformed_points_h = (transform @ points_h.T).T
    transformed_points = transformed_points_h[:, :-1]
    return transformed_points
    
# Load camera calibration
device = ase.get_ase_rgb_calibration()

# Load the trajectory using read_trajectory_file() 
trajectory_path = scene_path / "trajectory.csv"
trajectory = read_trajectory_file(trajectory_path)

# Load scene point cloud using read_points_file()
points_path = scene_path / "semidense_points.csv.gz"
points_world = read_points_file(points_path)

Loaded trajectory with 350 device poses.
Loaded point cloud with 433426 points.

# Transform the points into the device coordinate frame
T_world_from_device = trajectory["Ts_world_from_device"][frame_idx]
T_device_from_world = np.linalg.inv(T_world_from_device)
points_device = transform_3d_points(T_device_from_world, points_world)

# move from device to camera frame (device is device center but camera is offset)

# Transform the points into the RGB camera coordinate frame
T_device_from_camera = device.get_transform_device_camera().to_matrix()
T_camera_from_device = np.linalg.inv(T_device_from_camera)
points_device = transform_3d_points(T_camera_from_device, points_device)

# Project points into the image
points_image = []

points_device_filtered = []
depths = []
for idx, point_device in enumerate(points_device):
    point_image = device.project(point_device)
    if point_image is not None:
        points_image.append(point_image)
        points_device_filtered.append(point_device)
points_image = np.stack(points_image)

print (len (points_image))

# Overlay projected points onto image
plt.imshow(np.array(rgb))
# plt.scatter(points_image[:, 0], points_image[:, 1], s=0.01, alpha=0.3, c="#FFFFFF")
plt.axis("off")
plt.show()
print (np.array(rgb).shape)

plt.imshow(rgb)
plt.scatter(points_image[:, 0], points_image[:, 1], s=0.01, alpha=0.3, c="#FFFFFF")
plt.axis("off")
plt.show()

56286

(704, 704, 3)

points_image = np.array(points_image)
points_device_filtered = np.array(points_device_filtered)
depth_img = np.array(depth)

# make a 3D plot with the depthmap - pull that into 3D, and the camera frame z depth

x = np.arange(depth_img.shape[1])
y = np.arange(depth_img.shape[0])
xx, yy = np.meshgrid(x, y)

x_flat = xx.flatten()
y_flat = yy.flatten()
z_flat = depth_img.flatten()

x_flat_norm = (x_flat-(x_flat.max()/2)) # center
y_flat_norm = (y_flat-(y_flat.max()/2))


plot_traces = [go.Scatter3d(
    x=x_flat_norm,
    y=y_flat_norm,
    z=z_flat,
    mode="markers",
    name="rendered depth map",
    marker={
        "size": 1,
        "opacity": 0.2,
        "color": z_flat,
        "colorscale": "Viridis",
        "colorbar": {"title": "Depth"},
    }
),go.Scatter3d(
    x=points_image[:,0]-(points_image[:,0].max()/2),
    y=points_image[:,1]-(points_image[:,1].max()/2),
    z=points_device_filtered[:,2]*1000,
    mode="markers",
    name="point cloud",
    marker={
        "size": 1,
        "opacity": 0.2,
    }
)
]

fig = go.Figure(data=plot_traces)
fig.update_layout(
    height=600
)

fig.show()

Notebook stuck?¶

Part 1: Reading Scene Annotations and 3D Plotting¶

Part 2: Loading and Plotting Images and Image Annotations¶

Part 3: Project Points into Images¶